docs/source/tutorials/Preprocessing SIM with municipality.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "from pysus.preprocessing.geodata import add_data_to_municipality\n",
    "from pysus.preprocessing.SIM import group_and_count, redistribute_missing, redistribute_cid_chapter\n",
    "from pysus.preprocessing.decoders import translate_variables_SIM\n",
    "\n",
    "from pysus.ftp.databases.sim import SIM\n",
    "\n",
    "sim = SIM().load()\n",
    "\n",
    "states = [\"AC\", \"AM\", \"AP\", \"PA\", \"RO\", \"RR\", \"TO\"]\n",
    "variables = ['CODMUNRES','SEXO','IDADE_ANOS','CID10_CHAPTER']\n",
    "filter_variables = variables[:3]\n",
    "geo_df = pd.DataFrame()\n",
    "ages_args = {\n",
    "    \"start\": 0,\n",
    "    \"end\": 90,\n",
    "    \"freq\": 5\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████████████████████████████████████████████████████████████████████████████████████████████████| 330k/330k [00:00<00:00, 167MB/s]\n",
      "2023-09-19 13:53:06.107 | DEBUG    | pysus.online_data.SIM:get_municipios:185 - Stablishing connection with ftp.datasus.gov.br.\n",
      "220 Microsoft FTP Service\n",
      "2023-09-19 13:53:06.128 | DEBUG    | pysus.online_data.SIM:get_municipios:189 - Changing FTP work dir to: /dissemin/publicos/SIM/CID10/TABELAS\n",
      "2023-09-19 13:53:06.129 | INFO     | pysus.online_data.SIM:get_municipios:199 - Local parquet file found at /home/bida/pysus/SIM_CADMUN_.parquet\n",
      "2023-09-19 13:53:06.371 | DEBUG    | pysus.online_data.SIM:get_CID10_chapters_table:47 - Stablishing connection with ftp.datasus.gov.br.\n",
      "220 Microsoft FTP Service\n",
      "2023-09-19 13:53:06.392 | DEBUG    | pysus.online_data.SIM:get_CID10_chapters_table:51 - Changing FTP work dir to: /dissemin/publicos/SIM/CID10/TABELAS\n",
      "2023-09-19 13:53:06.393 | INFO     | pysus.online_data.SIM:get_CID10_chapters_table:61 - Local parquet file found at /home/bida/pysus/SIM_CIDCAP10_.parquet\n",
      "100%|███████████████████████████████████████████████████████████████████████████████████████████████████| 1.42M/1.42M [00:00<00:00, 1.07GB/s]\n",
      "2023-09-19 13:53:08.409 | DEBUG    | pysus.online_data.SIM:get_municipios:185 - Stablishing connection with ftp.datasus.gov.br.\n",
      "220 Microsoft FTP Service\n",
      "2023-09-19 13:53:08.431 | DEBUG    | pysus.online_data.SIM:get_municipios:189 - Changing FTP work dir to: /dissemin/publicos/SIM/CID10/TABELAS\n",
      "2023-09-19 13:53:08.432 | INFO     | pysus.online_data.SIM:get_municipios:199 - Local parquet file found at /home/bida/pysus/SIM_CADMUN_.parquet\n",
      "2023-09-19 13:53:08.879 | DEBUG    | pysus.online_data.SIM:get_CID10_chapters_table:47 - Stablishing connection with ftp.datasus.gov.br.\n",
      "220 Microsoft FTP Service\n",
      "2023-09-19 13:53:08.899 | DEBUG    | pysus.online_data.SIM:get_CID10_chapters_table:51 - Changing FTP work dir to: /dissemin/publicos/SIM/CID10/TABELAS\n",
      "2023-09-19 13:53:08.899 | INFO     | pysus.online_data.SIM:get_CID10_chapters_table:61 - Local parquet file found at /home/bida/pysus/SIM_CIDCAP10_.parquet\n",
      "100%|██████████████████████████████████████████████████████████████████████████████████████████████████████| 264k/264k [00:00<00:00, 173MB/s]\n",
      "2023-09-19 13:53:11.067 | DEBUG    | pysus.online_data.SIM:get_municipios:185 - Stablishing connection with ftp.datasus.gov.br.\n",
      "220 Microsoft FTP Service\n",
      "2023-09-19 13:53:11.088 | DEBUG    | pysus.online_data.SIM:get_municipios:189 - Changing FTP work dir to: /dissemin/publicos/SIM/CID10/TABELAS\n",
      "2023-09-19 13:53:11.089 | INFO     | pysus.online_data.SIM:get_municipios:199 - Local parquet file found at /home/bida/pysus/SIM_CADMUN_.parquet\n",
      "2023-09-19 13:53:11.311 | DEBUG    | pysus.online_data.SIM:get_CID10_chapters_table:47 - Stablishing connection with ftp.datasus.gov.br.\n",
      "220 Microsoft FTP Service\n",
      "2023-09-19 13:53:11.332 | DEBUG    | pysus.online_data.SIM:get_CID10_chapters_table:51 - Changing FTP work dir to: /dissemin/publicos/SIM/CID10/TABELAS\n",
      "2023-09-19 13:53:11.333 | INFO     | pysus.online_data.SIM:get_CID10_chapters_table:61 - Local parquet file found at /home/bida/pysus/SIM_CIDCAP10_.parquet\n",
      "100%|███████████████████████████████████████████████████████████████████████████████████████████████████| 3.22M/3.22M [00:00<00:00, 4.17GB/s]\n",
      "2023-09-19 13:53:13.178 | DEBUG    | pysus.online_data.SIM:get_municipios:185 - Stablishing connection with ftp.datasus.gov.br.\n",
      "220 Microsoft FTP Service\n",
      "2023-09-19 13:53:13.199 | DEBUG    | pysus.online_data.SIM:get_municipios:189 - Changing FTP work dir to: /dissemin/publicos/SIM/CID10/TABELAS\n",
      "2023-09-19 13:53:13.200 | INFO     | pysus.online_data.SIM:get_municipios:199 - Local parquet file found at /home/bida/pysus/SIM_CADMUN_.parquet\n",
      "2023-09-19 13:53:14.100 | DEBUG    | pysus.online_data.SIM:get_CID10_chapters_table:47 - Stablishing connection with ftp.datasus.gov.br.\n",
      "220 Microsoft FTP Service\n",
      "2023-09-19 13:53:14.122 | DEBUG    | pysus.online_data.SIM:get_CID10_chapters_table:51 - Changing FTP work dir to: /dissemin/publicos/SIM/CID10/TABELAS\n",
      "2023-09-19 13:53:14.123 | INFO     | pysus.online_data.SIM:get_CID10_chapters_table:61 - Local parquet file found at /home/bida/pysus/SIM_CIDCAP10_.parquet\n",
      "100%|██████████████████████████████████████████████████████████████████████████████████████████████████████| 665k/665k [00:00<00:00, 509MB/s]\n",
      "2023-09-19 13:53:19.139 | DEBUG    | pysus.online_data.SIM:get_municipios:185 - Stablishing connection with ftp.datasus.gov.br.\n",
      "220 Microsoft FTP Service\n",
      "2023-09-19 13:53:19.160 | DEBUG    | pysus.online_data.SIM:get_municipios:189 - Changing FTP work dir to: /dissemin/publicos/SIM/CID10/TABELAS\n",
      "2023-09-19 13:53:19.161 | INFO     | pysus.online_data.SIM:get_municipios:199 - Local parquet file found at /home/bida/pysus/SIM_CADMUN_.parquet\n",
      "2023-09-19 13:53:19.459 | DEBUG    | pysus.online_data.SIM:get_CID10_chapters_table:47 - Stablishing connection with ftp.datasus.gov.br.\n",
      "220 Microsoft FTP Service\n",
      "2023-09-19 13:53:19.481 | DEBUG    | pysus.online_data.SIM:get_CID10_chapters_table:51 - Changing FTP work dir to: /dissemin/publicos/SIM/CID10/TABELAS\n",
      "2023-09-19 13:53:19.482 | INFO     | pysus.online_data.SIM:get_CID10_chapters_table:61 - Local parquet file found at /home/bida/pysus/SIM_CIDCAP10_.parquet\n",
      "100%|██████████████████████████████████████████████████████████████████████████████████████████████████████| 230k/230k [00:00<00:00, 146MB/s]\n",
      "2023-09-19 13:53:21.358 | DEBUG    | pysus.online_data.SIM:get_municipios:185 - Stablishing connection with ftp.datasus.gov.br.\n",
      "220 Microsoft FTP Service\n",
      "2023-09-19 13:53:21.379 | DEBUG    | pysus.online_data.SIM:get_municipios:189 - Changing FTP work dir to: /dissemin/publicos/SIM/CID10/TABELAS\n",
      "2023-09-19 13:53:21.381 | INFO     | pysus.online_data.SIM:get_municipios:199 - Local parquet file found at /home/bida/pysus/SIM_CADMUN_.parquet\n",
      "2023-09-19 13:53:21.597 | DEBUG    | pysus.online_data.SIM:get_CID10_chapters_table:47 - Stablishing connection with ftp.datasus.gov.br.\n",
      "220 Microsoft FTP Service\n",
      "2023-09-19 13:53:21.618 | DEBUG    | pysus.online_data.SIM:get_CID10_chapters_table:51 - Changing FTP work dir to: /dissemin/publicos/SIM/CID10/TABELAS\n",
      "2023-09-19 13:53:21.619 | INFO     | pysus.online_data.SIM:get_CID10_chapters_table:61 - Local parquet file found at /home/bida/pysus/SIM_CIDCAP10_.parquet\n",
      "100%|██████████████████████████████████████████████████████████████████████████████████████████████████████| 659k/659k [00:00<00:00, 554MB/s]\n",
      "2023-09-19 13:53:22.812 | DEBUG    | pysus.online_data.SIM:get_municipios:185 - Stablishing connection with ftp.datasus.gov.br.\n",
      "220 Microsoft FTP Service\n",
      "2023-09-19 13:53:22.833 | DEBUG    | pysus.online_data.SIM:get_municipios:189 - Changing FTP work dir to: /dissemin/publicos/SIM/CID10/TABELAS\n",
      "2023-09-19 13:53:22.834 | INFO     | pysus.online_data.SIM:get_municipios:199 - Local parquet file found at /home/bida/pysus/SIM_CADMUN_.parquet\n",
      "2023-09-19 13:53:23.138 | DEBUG    | pysus.online_data.SIM:get_CID10_chapters_table:47 - Stablishing connection with ftp.datasus.gov.br.\n",
      "220 Microsoft FTP Service\n",
      "2023-09-19 13:53:23.159 | DEBUG    | pysus.online_data.SIM:get_CID10_chapters_table:51 - Changing FTP work dir to: /dissemin/publicos/SIM/CID10/TABELAS\n",
      "2023-09-19 13:53:23.161 | INFO     | pysus.online_data.SIM:get_CID10_chapters_table:61 - Local parquet file found at /home/bida/pysus/SIM_CIDCAP10_.parquet\n"
     ]
    }
   ],
   "source": [
    "import warnings\n",
    "warnings.filterwarnings('ignore')\n",
    "\n",
    "for state in states:\n",
    "    df = sim.download(sim.get_files(\"CID10\",state,2018)).to_dataframe()\n",
    "    df = translate_variables_SIM(df,age_classes=True,classify_args=ages_args,classify_cid10_chapters=True)\n",
    "    df = df[variables]\n",
    "    counts = group_and_count(df,variables)\n",
    "    counts = redistribute_missing(counts,filter_variables)\n",
    "    counts = redistribute_cid_chapter(counts,filter_variables)\n",
    "    geo_df = geo_df.append(add_data_to_municipality(counts,title_cols=['SEXO','IDADE_ANOS','CID10_CHAPTER'], nan_string='NA'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Index(['code_muni', 'name_muni', 'code_state', 'abbrev_state', 'name_state',\n",
       "       'code_region', 'name_region', 'geometry', 'Feminino-[0.0, 5.0)-1',\n",
       "       'Feminino-[0.0, 5.0)-2',\n",
       "       ...\n",
       "       'NA-[50.0, 55.0)-8', 'NA-[55.0, 60.0)-8', 'NA-[60.0, 65.0)-8',\n",
       "       'NA-[65.0, 70.0)-8', 'NA-[70.0, 75.0)-8', 'NA-[75.0, 80.0)-8',\n",
       "       'NA-[80.0, 85.0)-8', 'NA-[85.0, 90.0)-8', 'NA-[90.0, inf)-8',\n",
       "       'NA-NA-8'],\n",
       "      dtype='object', length=1088)"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "geo_df.columns"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.5"
  },
  "vscode": {
   "interpreter": {
    "hash": "2a96a5ccec8dfcba7d06b2e71f6eef3b5dac5716461bf5d73ea1bb7ee462cdaa"
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}